require 'config/boot'
require 'config/environment'
text = ENV['text']

task :encode do
  File.open(text).each do |line|
    line.strip!
    array = line.split(/\t/)
    print array[3].to_s
    if array[3].to_s == "。"
      print "\n"
    end
  end
end

task :check_code do
  UTF8 = /\A(?:
   [\x00-\x7f]                                     |
   [\xc2-\xdf] [\x80-\xbf]                         |
   \xe0        [\xa0-\xbf] [\x80-\xbf]             |
   [\xe1-\xef] [\x80-\xbf] [\x80-\xbf]             |
   \xf0        [\x90-\xbf] [\x80-\xbf] [\x80-\xbf] |
   [\xf1-\xf3] [\x80-\xbf] [\x80-\xbf] [\x80-\xbf] |
   \xf4        [\x80-\x8f] [\x80-\xbf] [\x80-\xbf]
  )*\z/nx
  
  File.open(text).each do |line|
    line.strip!
    print "error" if UTF8.match(line).blank?
  end
end